{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "name": "document_normalizer_demo.ipynb",
   "provenance": [],
   "collapsed_sections": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  }
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uXIWSN300w5v"
   },
   "source": [
    "\n",
    "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)\n",
    "\n",
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/nlu/blob/master/examples/colab/component_examples/text_pre_processing_and_cleaning/document_normalizer_demo.ipynb)\n",
    "\n",
    "\n",
    "\n",
    "The DocumentNormalizer extracts content from HTML or XML documents, applying either data cleansing using an arbitrary number of custom regular expressions either data extraction following the different parameters"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "abuB9K1_QVuL",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1649990022674,
     "user_tz": -300,
     "elapsed": 96885,
     "user": {
      "displayName": "ahmed lone",
      "userId": "02458088882398909889"
     }
    },
    "outputId": "28f0d04f-dade-464a-c1ef-4840f55dd381"
   },
   "source": [
    "!wget https://setup.johnsnowlabs.com/nlu/colab.sh -O - | bash\n",
    "  \n",
    "\n",
    "import nlu"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "--2022-04-15 02:32:04--  https://setup.johnsnowlabs.com/nlu/colab.sh\n",
      "Resolving setup.johnsnowlabs.com (setup.johnsnowlabs.com)... 51.158.130.125\n",
      "Connecting to setup.johnsnowlabs.com (setup.johnsnowlabs.com)|51.158.130.125|:443... connected.\n",
      "HTTP request sent, awaiting response... 302 Moved Temporarily\n",
      "Location: https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/scripts/colab_setup.sh [following]\n",
      "--2022-04-15 02:32:05--  https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/scripts/colab_setup.sh\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.109.133, 185.199.108.133, ...\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 1665 (1.6K) [text/plain]\n",
      "Saving to: ‘STDOUT’\n",
      "\n",
      "-                   100%[===================>]   1.63K  --.-KB/s    in 0s      \n",
      "\n",
      "2022-04-15 02:32:05 (31.9 MB/s) - written to stdout [1665/1665]\n",
      "\n",
      "Installing  NLU 3.4.3rc2 with  PySpark 3.0.3 and Spark NLP 3.4.2 for Google Colab ...\n",
      "Ign:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
      "Get:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]\n",
      "Ign:3 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
      "Get:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release [696 B]\n",
      "Hit:5 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
      "Get:6 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release.gpg [836 B]\n",
      "Get:7 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease [15.9 kB]\n",
      "Hit:8 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
      "Get:9 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
      "Get:10 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
      "Hit:11 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
      "Get:13 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease [15.9 kB]\n",
      "Get:14 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
      "Get:15 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Packages [953 kB]\n",
      "Hit:16 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
      "Get:17 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main Sources [1,947 kB]\n",
      "Get:18 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [2,695 kB]\n",
      "Get:19 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [3,134 kB]\n",
      "Get:20 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,490 kB]\n",
      "Get:21 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic/main amd64 Packages [996 kB]\n",
      "Get:22 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,268 kB]\n",
      "Get:23 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic/main amd64 Packages [45.3 kB]\n",
      "Fetched 13.8 MB in 4s (3,080 kB/s)\n",
      "Reading package lists... Done\n",
      "tar: spark-3.0.2-bin-hadoop2.7.tgz: Cannot open: No such file or directory\n",
      "tar: Error is not recoverable: exiting now\n",
      "\u001B[K     |████████████████████████████████| 209.1 MB 55 kB/s \n",
      "\u001B[K     |████████████████████████████████| 142 kB 68.0 MB/s \n",
      "\u001B[K     |████████████████████████████████| 505 kB 51.1 MB/s \n",
      "\u001B[K     |████████████████████████████████| 198 kB 69.1 MB/s \n",
      "\u001B[?25h  Building wheel for pyspark (setup.py) ... \u001B[?25l\u001B[?25hdone\n",
      "Requirement already satisfied: nlu_tmp==3.4.3rc10 in /usr/local/lib/python3.7/dist-packages (3.4.3rc10)\n",
      "Requirement already satisfied: pandas>=1.3.5 in /usr/local/lib/python3.7/dist-packages (from nlu_tmp==3.4.3rc10) (1.3.5)\n",
      "Requirement already satisfied: pyarrow>=0.16.0 in /usr/local/lib/python3.7/dist-packages (from nlu_tmp==3.4.3rc10) (6.0.1)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from nlu_tmp==3.4.3rc10) (1.21.5)\n",
      "Requirement already satisfied: dataclasses in /usr/local/lib/python3.7/dist-packages (from nlu_tmp==3.4.3rc10) (0.6)\n",
      "Requirement already satisfied: spark-nlp<3.5.0,>=3.4.2 in /usr/local/lib/python3.7/dist-packages (from nlu_tmp==3.4.3rc10) (3.4.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.3.5->nlu_tmp==3.4.3rc10) (2018.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.3.5->nlu_tmp==3.4.3rc10) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas>=1.3.5->nlu_tmp==3.4.3rc10) (1.15.0)\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4SQzdgwyQ65Q",
    "executionInfo": {
     "status": "ok",
     "timestamp": 1649990116154,
     "user_tz": -300,
     "elapsed": 704,
     "user": {
      "displayName": "ahmed lone",
      "userId": "02458088882398909889"
     }
    },
    "outputId": "cd64e425-eed6-47aa-d78f-0f0aab897829"
   },
   "source": [
    "pipe = nlu.load('norm_document')\n",
    "data = '<!DOCTYPE html> <html> <head> <title>Example</title> </head> <body> <p>This is an example of a simple HTML page with one paragraph.</p> </body> </html>'\n",
    "df = pipe.predict(data,output_level='document')\n",
    "print(df['norm'])\n",
    "print(df.iloc[0]['norm'])"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "0    [DOCTYPE, html, html, head, titleExampletitle,...\n",
      "Name: norm, dtype: object\n",
      "['DOCTYPE', 'html', 'html', 'head', 'titleExampletitle', 'head', 'body', 'pThis', 'is', 'an', 'example', 'of', 'a', 'simple', 'HTML', 'page', 'with', 'one', 'paragraphp', 'body', 'html']\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "o73dDzocR7L_",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "status": "ok",
     "timestamp": 1649990128424,
     "user_tz": -300,
     "elapsed": 1280,
     "user": {
      "displayName": "ahmed lone",
      "userId": "02458088882398909889"
     }
    },
    "outputId": "1a30273a-2c28-4158-b20d-410e13cdf23d"
   },
   "source": [
    "pipe.print_info()\n"
   ],
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "The following parameters are configurable for this NLU pipeline (You can copy paste the examples) :\n",
      ">>> component_list['normalizer'] has settable params:\n",
      "component_list['normalizer'].setCleanupPatterns(['[^\\\\pL+]'])  | Info: normalization regex patterns which match will be removed from token | Currently set to : ['[^\\\\pL+]']\n",
      "component_list['normalizer'].setLowercase(False)               | Info: whether to convert strings to lowercase | Currently set to : False\n",
      "component_list['normalizer'].setSlangMatchCase(False)          | Info: whether or not to be case sensitive to match slangs. Defaults to false. | Currently set to : False\n",
      "component_list['normalizer'].setMinLength(0)                   | Info: Set the minimum allowed legth for each token | Currently set to : 0\n",
      ">>> component_list['tokenizer'] has settable params:\n",
      "component_list['tokenizer'].setTargetPattern('\\S+')            | Info: pattern to grab from text as token candidates. Defaults \\S+ | Currently set to : \\S+\n",
      "component_list['tokenizer'].setContextChars(['.', ',', ';', ':', '!', '?', '*', '-', '(', ')', '\"', \"'\"])  | Info: character list used to separate from token boundaries | Currently set to : ['.', ',', ';', ':', '!', '?', '*', '-', '(', ')', '\"', \"'\"]\n",
      "component_list['tokenizer'].setCaseSensitiveExceptions(True)   | Info: Whether to care for case sensitiveness in exceptions | Currently set to : True\n",
      "component_list['tokenizer'].setMinLength(0)                    | Info: Set the minimum allowed legth for each token | Currently set to : 0\n",
      "component_list['tokenizer'].setMaxLength(99999)                | Info: Set the maximum allowed legth for each token | Currently set to : 99999\n",
      ">>> component_list['document_assembler'] has settable params:\n",
      "component_list['document_assembler'].setCleanupMode('shrink')  | Info: possible values: disabled, inplace, inplace_full, shrink, shrink_full, each, each_full, delete_full | Currently set to : shrink\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "id": "kRpnTBViEfP4"
   },
   "execution_count": null,
   "outputs": []
  }
 ]
}
